package com.mxw.service.impl;

import com.mxw.service.WebCrawlerService;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.springframework.stereotype.Service;

import java.io.IOException;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

@Service
public class WebCrawlerServiceImpl implements WebCrawlerService {

    // 匹配中文和数字的正则表达式
    private static final Pattern CHINESE_AND_NUMBER_PATTERN = Pattern.compile("[\\u4e00-\\u9fa5]+|\\d+");

    @Override
    public String getWebContent(String url) throws IOException {
        // 设置超时时间并获取网页内容
        Document doc = Jsoup.connect(url)
                .timeout(10000)
                .userAgent("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36")
                .get();

        // 移除不需要的元素
        doc.select("script, style, iframe, nav, footer, header, .footer, .header, .nav, .sidebar, .comment, .ad, .advertisement").remove();

        // 优先获取主要内容区域
        StringBuilder content = new StringBuilder();
        
        // 获取标题
        String title = doc.title();
        if (title != null && !title.isEmpty()) {
            content.append("标题：").append(title).append("\n\n");
        }

        // 按优先级尝试获取主要内容
        Element mainContent = null;
        
        // 1. 尝试获取article标签
        mainContent = doc.selectFirst("article");
        
        // 2. 如果没有article，尝试获取主要内容区域
        if (mainContent == null) {
            mainContent = doc.selectFirst(".article, .content, .post, .entry, .main, #content, #main, .main-content");
        }
        
        // 3. 如果还是没有找到主要内容区域，使用body
        if (mainContent == null) {
            mainContent = doc.body();
        }

        // 获取文本内容
        String text = mainContent.text();
        
        // 提取所有中文和数字
        Matcher matcher = CHINESE_AND_NUMBER_PATTERN.matcher(text);
        while (matcher.find()) {
            content.append(matcher.group()).append(" ");
        }

        return content.toString().trim();
    }



} 